package com.omg.server.crawler.strategy.impl;

import java.net.URLEncoder;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;

import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.Source;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.omg.business.model.PostMeta;
import com.omg.business.model.Posts;
import com.omg.business.model.Terms;
import com.omg.business.model.TermsRelation;
import com.omg.business.model.TermsTaxonomy;
import com.omg.server.crawler.RequestContext;
import com.omg.server.crawler.strategy.CrawlerStrategy;
import com.omg.server.framework.service.IDefaultService;
import com.omg.server.util.CommonUtil;
import com.omg.server.util.DateUtil;
import com.omg.server.util.HttpSenderUtil;
import com.omg.weibo.WeiBoSender;

/**
 * 关于什么值得买 网站抓取策略
 * @author Administrator
 */
public class ZbzdmStrategyImpl implements CrawlerStrategy{
	private static final Log log = LogFactory.getLog(ZbzdmStrategyImpl.class);
	private IDefaultService defaultService;
	@Override
	public void handle(RequestContext requestContext) {
		int pageIndex = requestContext.getStartIndex();
		if (pageIndex < 1){
			return;
		}
		String url = requestContext.getSiteUrl();
		String soureUrl = requestContext.getSiteUrl();
		log.info(url + "第" + pageIndex + "抓取.");
		
		Source source = null;
		Source source2 = null;
		String[] cataTypes = new String[]{"1","2"};;
		for(String cataType : cataTypes){
			try {
				String subStract = null;
				String imgUrl = null;
				String imgUrl2 = null;
				String targetUrl = null;
				Element bizElement = null;
				Element subElement = null;
				Element subElement2 = null;
				String relativeUrl = null;
				Posts posts = null;
				PostMeta postMeta = null;
				PostMeta postMeta2 = null;
				List<Element> bizList = null;
				String price = null;
				String title = null;
				String issueTime = null;
				source = HttpSenderUtil.getStreamByUrl(url+"bbs/portal.php?mod=list&catid="+cataType+"&page="+pageIndex);
				// Call fullSequentialParse manually as most of the source will be parsed.
				source.fullSequentialParse();
				String pageUrl = null;
				List<Element> content = null;
				//bizList = source.getAllElementsByClass("xs2 cl");
				bizList = source.getAllElements(HTMLElementName.DD);
				Date issueDate = null;
				List<Integer> count = null;
				String type = null;
				String tag = null;
				Map<String, Object> params = new HashMap<String, Object>(2);
				HashSet<PostMeta> pmSet = new HashSet<PostMeta>(2);
				if (CommonUtil.isNotEmpty(bizList)){
					WeiBoSender wbs = new WeiBoSender();
					for (Element el : bizList){
						try{
							title = null;
							price = null;
							subStract = null;
							targetUrl = null;
							issueDate = null;
							type = null;
							imgUrl = null;
							imgUrl2 = null;
							issueTime = null;
							pageUrl = null;
							relativeUrl = null;
							posts = new Posts();
							postMeta = new PostMeta();
							postMeta2 = new PostMeta();
							subElement = el.getFirstElementByClass("atc");
							if (null != subElement){
								subElement = subElement.getFirstElement(HTMLElementName.IMG);
								if (null != subElement){
									imgUrl = subElement.getAttributeValue("src");//商品图片地址
									if (null !=imgUrl && imgUrl.indexOf("http:") < 0){
										if (imgUrl.startsWith("/")){
										   imgUrl = soureUrl.substring(0, soureUrl.length()-1) +"bbs"+ imgUrl;
										}
										else{
											imgUrl = soureUrl + "bbs/" +imgUrl;
										}
									}
								}
							}
							subElement = el.getFirstElementByClass("xi2");
							if (null != subElement){
								//分离出价格
								title = subElement.getTextExtractor().toString();
								price = CommonUtil.getPrice(title);
								posts.setPostTitle(title);
								if (null == price){
									price = "";
								}
								posts.setPrice(price);
							}
							if ("1".equals(cataType)){//国内优惠
								subElement = el.getFirstElement("label");
								if (null == subElement){
									continue;
								}
								
								subElement2 = subElement.getFirstElementByClass("xi2");
								if (null != subElement2){
									type = subElement2.getTextExtractor().toString();
									if (CommonUtil.isNotEmpty(type)){
										type = type.trim();
									}
								}
							}
							else{
								type = "国外优惠";
							}
							
							subElement2 = el.getFirstElementByClass("xg1");
							if (null != subElement2){
								//自动加年月日
								issueTime = subElement2.getContent().toString();
								if (CommonUtil.isNotEmpty(issueTime)){
									issueTime = issueTime.trim();
								}
								else{
									continue;
								}
								issueDate = DateUtil.convertToDate(issueTime, DateUtil.PATTERN_NOSECOND);
								log.debug("zbzdm issueDate:" + issueDate);
								System.out.println(issueDate);
								if (null != requestContext.getMaxDate() && null != issueDate){
									if (issueDate.before(requestContext.getMaxDate())){
										continue;
									}
								}
								posts.setPostDate(issueDate);
								posts.setPostDate2(issueDate);
								posts.setPostModified(issueDate);
								posts.setPostModified2(issueDate);
							}
							
							content = el.getAllElements(HTMLElementName.A);
							if (null != content){
								bizElement = content.get(content.size()-1);
								relativeUrl = bizElement.getAttributeValue("href");
								if (null != relativeUrl && relativeUrl.startsWith("/")){
									relativeUrl = relativeUrl.substring(1);
								}
								pageUrl = url + "bbs/"+ relativeUrl;
								source2 = HttpSenderUtil.getStreamByUrl(pageUrl);
								if (null != source2)
								bizElement = source2.getFirstElementByClass("vwtb");
								if (null != bizElement){
								    subElement = bizElement.getFirstElement(HTMLElementName.A);
								    if (null != subElement){
								        targetUrl = subElement.getAttributeValue("href");//商品目标来源
								    }
								    subStract = bizElement.getTextExtractor().toString();
								
								    //明细中的图片
									List<Element> imgsList = bizElement.getAllElements(HTMLElementName.IMG);
									if(null != imgsList)
									for (Element e : imgsList){
										if (CommonUtil.isEmpty(imgUrl2)){
											imgUrl2 = e.getAttributeValue("src");
											if (null !=imgUrl2 && imgUrl2.indexOf("http:") < 0){
												if (imgUrl2.startsWith("/")){
													imgUrl2 = soureUrl.substring(0, soureUrl.length()-1) + "bbs" + imgUrl2;
												}
												else{
													imgUrl2 = soureUrl + "bbs/" +imgUrl2;
												}
											}
										}
										subStract = subStract.replace(e.toString(), "");
									}
								}
								
								if(CommonUtil.isEmpty(imgUrl)){
									imgUrl = imgUrl2;
								}
								
								targetUrl = CommonUtil.getFinalUrl(targetUrl);
								if (null != targetUrl){
									if (targetUrl.contains(soureUrl)){
										continue;
									}
									if (targetUrl.startsWith("ttp://")){
										targetUrl = "h" + targetUrl;
									}
								    //targetUrl = URLDecoder.decode(targetUrl);
								}
								
								//检查库中是否有这样的数据
								params.put("title", title);
								params.put("soureUrl", soureUrl);
								params.put("targetUrl", targetUrl);
								count = defaultService.find_iBatis("omg.crawler.checkGoodsIsExist", params);
								if (CommonUtil.isNotEmpty(count)){
									if (count.get(0) > 0){
										continue;
									}
								}
				
								
								postMeta.setMetaKey("thumbnail");
								postMeta.setMetaValue(imgUrl);
								postMeta2.setMetaKey("targeturl");
								postMeta2.setMetaValue(targetUrl);
								//postMeta.setId(UUID.randomUUID().toString());
								//postMeta2.setId(UUID.randomUUID().toString());
								posts.setSourceUrl(url);
								posts.setPostContent(subStract);
								posts.setPostName(title);
								
								//int postId = 0;
								if (CommonUtil.isNotEmpty(title) && CommonUtil.isNotEmpty(subStract) && CommonUtil.isNotEmpty(targetUrl)){
									pmSet.add(postMeta);
									pmSet.add(postMeta2);
									//posts.setPostMetas(pmSet);
									WriteDBUtil.writeToDb(posts, pmSet, type, tag, ",", ",", imgUrl, wbs);
								   // defaultService.save_Hibernate_Object(posts);
								    pmSet.clear(); //清空扩展
								    /*
								    postId = posts.getId();
								    if (postId <= 0){
								    	log.debug("生成的post_id 为:" + postId);
								    }
								    log.debug("post_id:" + postId);
								    this.saveCataAndTag(type, tag, postId);
								    if ("1".equals(PropertiesHelper.getProperty(ISSENDWEIBO))){
								        //发送微博
								        wbs.send(title, imgUrl, CrawlerStrategy.SITEURL+postId);
								    }*/
								}
								log.info(url + "商品录入成功.");
						   }
					  }
					  catch (Exception e) {
							e.printStackTrace();
							log.error("录入商品失败", e);
					  }
					  finally{
							  pmSet.clear(); //清空扩展
					  }
				   }
				}
			}
			catch (Exception e) {
				log.error(e);
				e.printStackTrace();
			}
		}
	}
	
    private void saveCataAndTag(String type, String tag, int postId){
    	Map<String, String> params = new HashMap<String,String>();
    	List<Integer> termTaxonomy = null;
    	int cataTermTaxonomyId = -1;
    	int tagTermTaxonomyId = -1;
    	
    	if (CommonUtil.isEmpty(type)){
			type = "其它";
		}
		else{
			type = type.trim();
		}
		
		
		String[] types = type.split(",");
		TermsRelation termsRelation = null;
		TermsTaxonomy termsTaxonomy = null;
		Terms terms = null;
		for (int i=0, len=types.length; i < len; i++){
			cataTermTaxonomyId = -1;
			//检查目录是否存在
			params.clear();
			params.put("taxonomy", "category");
			params.put("termName", types[i]);
			termTaxonomy = defaultService.find_iBatis("omg.post.checkTermsIsExist", params);
			if (CommonUtil.isNotEmpty(termTaxonomy)){
				cataTermTaxonomyId = termTaxonomy.get(0);
			}
			
			//保存分类目录
			termsRelation = new TermsRelation();
			if (-1 == cataTermTaxonomyId){
				terms = new Terms();
				terms.setName(types[i]);
				terms.setSlug(URLEncoder.encode(types[i]));
				defaultService.save_Hibernate_Object(terms);
				termsTaxonomy = new TermsTaxonomy();
				termsTaxonomy.setTermId(terms.getId());
				termsTaxonomy.setTaxonomy("category");
				termsTaxonomy.setDescription("商品分类目录");
				defaultService.save_Hibernate_Object(termsTaxonomy);
				cataTermTaxonomyId = termsTaxonomy.getId();
			}
			termsRelation.setId(postId);
			termsRelation.setTermOrder(i);
			termsRelation.setTermTaxonomyId(cataTermTaxonomyId);
			defaultService.save_Hibernate_Object(termsRelation);
			log.debug("商品分类目录成功:" +postId);
		}
		if (CommonUtil.isNotEmpty(tag)){
			String[] tags = tag.split(",");
			for (int i=0,len=tags.length;i < len; i++){
				try{
					tagTermTaxonomyId = -1;
					//检查标签是否已存在
					params.clear();
					params.put("taxonomy", "post_tag");
					params.put("termName", tags[i]);
					termTaxonomy = defaultService.find_iBatis("omg.post.checkTermsIsExist", params);
					if (CommonUtil.isNotEmpty(termTaxonomy)){
						tagTermTaxonomyId = termTaxonomy.get(0);
					}
					
					//保存标签
					termsRelation = new TermsRelation();
					if (-1 == tagTermTaxonomyId){
					   terms = new Terms();
					   terms.setName(tags[i]);
					   terms.setSlug(URLEncoder.encode(tags[i]));
					   defaultService.save_Hibernate_Object(terms);
					   termsTaxonomy = new TermsTaxonomy();
					   termsTaxonomy.setTermId(terms.getId());
					   termsTaxonomy.setTaxonomy("post_tag");
					   termsTaxonomy.setDescription("商品标签");
					   defaultService.save_Hibernate_Object(termsTaxonomy);
					   tagTermTaxonomyId = termsTaxonomy.getId();
					}
					termsRelation.setId(postId);
					termsRelation.setTermOrder(i);
					termsRelation.setTermTaxonomyId(tagTermTaxonomyId);
					defaultService.save_Hibernate_Object(termsRelation);
					log.debug("商品标签成功:" +postId);
				}
				catch(Exception e){
					log.warn(e);
				}
			}
		}
    }
	

	public IDefaultService getDefaultService() {
		return defaultService;
	}

	public void setDefaultService(IDefaultService defaultService) {
		this.defaultService = defaultService;
	}
	public static void main(String[] args){
		RequestContext requestContext = new RequestContext();
		requestContext.setStartIndex(3);
		requestContext.setSiteUrl("http://www.zbzdm.com/");
		new ZbzdmStrategyImpl().handle(requestContext);
	}
	
}
